{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionnary_category_ability_paths = pickle.load(\n",
    "    open( \"dictionnary_category_ability_paths.pkl\", \"rb\")\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save it as json\n",
    "with open('dictionnary_category_ability_paths.json', 'w') as fp:\n",
    "    json.dump(dictionnary_category_ability_paths, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mkdir ../ressources/fonts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp -r /home/rbaena/Downloads/fonts-main/ ../ressources/fonts/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionnary_category_ability_paths = pickle.load(\n",
    "    open( \"dictionnary_category_ability_paths.pkl\", \"rb\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp -r /home/rbaena/projects/docExtractor/synthetic_resource/font/ resources/fonts/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_path = \"/home/rbaena/datasets\"\n",
    "import os\n",
    "import pickle\n",
    "with open(os.path.join(datasets_path, \"list_files.pkl\"), \"rb\") as f:\n",
    "    list_of_files = pickle.load(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save list_of_files as json in HWDB_v1\n",
    "import json\n",
    "with open(os.path.join(datasets_path, \"HWDB_v1\", \"list_files.json\"), \"w\") as f:\n",
    "    json.dump(list_of_files, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(datasets_path, \"HWDB_v1\", \"list_files.json\"), \"r\") as f:\n",
    "    list_of_files = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/rbaena/datasets/HWDB_v1/list_files.json'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.join(datasets_path, \"HWDB_v1\", \"list_files.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "/home/rbaena/datasets/HWDB_v1/list_files.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " enerating synthetic image 80/100/100 Generating synthetic image 2/100Generating synthetic image 3/100Generating synthetic image 6/100   Generating synthetic image 8/100Generating synthetic image 9/100   Generating synthetic image 10/100     Generating synthetic image 12/100 Generating synthetic image 14/100 Generating synthetic image 18/100   Generating synthetic image 27/100 Generating synthetic image 19/100Generating synthetic image 20/100 Generating synthetic image 28/100 Generating synthetic image 23/100Generating synthetic image 24/100Generating synthetic image 26/100Generating synthetic image 25/100Generating synthetic image 13/100Generating synthetic image 16/100Generating synthetic image 29/100Generating synthetic image 30/100Generating synthetic image 15/100Generating synthetic image 31/100Generating synthetic image 17/100Generating synthetic image 32/100 Generating synthetic image 22/100Generating synthetic image 33/100Generating synthetic image 34/100    Generating synthetic image 39/100Generating synthetic image 35/100Generating synthetic image 38/100Generating synthetic image 40/100Generating synthetic image 42/100Generating synthetic image 41/100Generating synthetic image 43/100Generating synthetic image 44/100  Generating synthetic image 47/100Generating synthetic image 48/100  Generating synthetic image 51/100Generating synthetic image 52/100 Generating synthetic image 54/100Generating synthetic image 55/100 Generating synthetic image 57/100Generating synthetic image 58/100 Generating synthetic image 56/100Generating synthetic image 59/100Generating synthetic image 60/100  Generating synthetic image 61/100 Generating synthetic image 63/100 Generating synthetic image 64/100   Generating synthetic image 68/100Generating synthetic image 67/100Generating synthetic image 65/100Generating synthetic image 66/100 Generating synthetic image 69/100Generating synthetic image 71/100 Generating synthetic image 70/100 Generating synthetic image 72/100Generating synthetic image 74/100Generating synthetic image 75/100Generating synthetic image 76/100Generating synthetic image 77/100 Generating synthetic image 79/100 Generating synthetic image 82/100 Generating synthetic image 81/100 Generating synthetic image 83/100 Generating synthetic image 84/100 Generating synthetic image 86/100Generating synthetic image 85/100Generating synthetic image 87/100Generating synthetic image 88/100  Generating synthetic image 91/100Generating synthetic image 89/100 Generating synthetic image 90/100Generating synthetic image 93/100 Generating synthetic image 94/100Generating synthetic image 92/100Generating synthetic image 95/100Generating synthetic image 96/100    Generating synthetic image 99/100Generating synthetic image 100/100Generating synthetic image 97/100Generating synthetic image 98/100"
     ]
    }
   ],
   "source": [
    "dataset_val.generates_synthetic_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_val.generates_synthetic_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1107-f.gnt',\n",
       " '568-f.gnt',\n",
       " '675-f.gnt',\n",
       " '129-f.gnt',\n",
       " '1230-f.gnt',\n",
       " '539-f.gnt',\n",
       " '1034-f.gnt',\n",
       " '283-f.gnt',\n",
       " '607-f.gnt',\n",
       " '411-f.gnt',\n",
       " '547-f.gnt',\n",
       " '301-f.gnt',\n",
       " '406-f.gnt',\n",
       " '030-f.gnt',\n",
       " '1042-f.gnt',\n",
       " '1094-f.gnt',\n",
       " '617-f.gnt',\n",
       " '668-f.gnt',\n",
       " '1061-f.gnt',\n",
       " '095-f.gnt',\n",
       " '244-f.gnt',\n",
       " '1212-f.gnt',\n",
       " '1098-f.gnt',\n",
       " '308-f.gnt',\n",
       " '262-f.gnt',\n",
       " '102-f.gnt',\n",
       " '232-f.gnt',\n",
       " '1043-f.gnt',\n",
       " '1233-f.gnt',\n",
       " '130-f.gnt',\n",
       " '733-f.gnt',\n",
       " '553-f.gnt',\n",
       " '1217-f.gnt',\n",
       " '614-f.gnt',\n",
       " '732-f.gnt',\n",
       " '1222-f.gnt',\n",
       " '285-f.gnt',\n",
       " '381-f.gnt',\n",
       " '698-f.gnt',\n",
       " '1078-f.gnt',\n",
       " '695-f.gnt',\n",
       " '1140-f.gnt',\n",
       " '1114-f.gnt',\n",
       " '615-f.gnt',\n",
       " '1051-f.gnt',\n",
       " '551-f.gnt',\n",
       " '335-f.gnt',\n",
       " '1172-f.gnt',\n",
       " '118-f.gnt',\n",
       " '116-f.gnt',\n",
       " '585-f.gnt',\n",
       " '346-f.gnt',\n",
       " '408-f.gnt',\n",
       " '593-f.gnt',\n",
       " '688-f.gnt',\n",
       " '363-f.gnt',\n",
       " '1102-f.gnt',\n",
       " '510-f.gnt',\n",
       " '1161-f.gnt',\n",
       " '257-f.gnt',\n",
       " '1017-f.gnt',\n",
       " '1131-f.gnt',\n",
       " '142-f.gnt',\n",
       " '174-f.gnt',\n",
       " '599-f.gnt',\n",
       " '563-f.gnt',\n",
       " '1200-f.gnt',\n",
       " '1022-f.gnt',\n",
       " '1129-f.gnt',\n",
       " '050-f.gnt',\n",
       " '086-f.gnt',\n",
       " '611-f.gnt',\n",
       " '714-f.gnt',\n",
       " '641-f.gnt',\n",
       " '032-f.gnt',\n",
       " '1214-f.gnt',\n",
       " '1167-f.gnt',\n",
       " '1137-f.gnt',\n",
       " '373-f.gnt',\n",
       " '582-f.gnt',\n",
       " '597-f.gnt',\n",
       " '337-f.gnt',\n",
       " '583-f.gnt',\n",
       " '312-f.gnt',\n",
       " '071-f.gnt',\n",
       " '700-f.gnt',\n",
       " '388-f.gnt',\n",
       " '708-f.gnt',\n",
       " '1007-f.gnt',\n",
       " '594-f.gnt',\n",
       " '336-f.gnt',\n",
       " '137-f.gnt',\n",
       " '1124-f.gnt',\n",
       " '024-f.gnt',\n",
       " '1047-f.gnt',\n",
       " '1199-f.gnt',\n",
       " '1236-f.gnt',\n",
       " '076-f.gnt',\n",
       " '629-f.gnt',\n",
       " '1085-f.gnt',\n",
       " '1135-f.gnt',\n",
       " '1002-f.gnt',\n",
       " '524-f.gnt',\n",
       " '682-f.gnt',\n",
       " '1187-f.gnt',\n",
       " '715-f.gnt',\n",
       " '126-f.gnt',\n",
       " '295-f.gnt',\n",
       " '567-f.gnt',\n",
       " '323-f.gnt',\n",
       " '1188-f.gnt',\n",
       " '224-f.gnt',\n",
       " '252-f.gnt',\n",
       " '595-f.gnt',\n",
       " '112-f.gnt',\n",
       " '1053-f.gnt',\n",
       " '005-f.gnt',\n",
       " '1040-f.gnt',\n",
       " '161-f.gnt',\n",
       " '304-f.gnt',\n",
       " '053-f.gnt',\n",
       " '022-f.gnt',\n",
       " '516-f.gnt',\n",
       " '352-f.gnt',\n",
       " '509-f.gnt',\n",
       " '725-f.gnt',\n",
       " '654-f.gnt',\n",
       " '511-f.gnt',\n",
       " '680-f.gnt',\n",
       " '673-f.gnt',\n",
       " '1019-f.gnt',\n",
       " '1012-f.gnt',\n",
       " '667-f.gnt',\n",
       " '1068-f.gnt',\n",
       " '635-f.gnt',\n",
       " '1010-f.gnt',\n",
       " '008-f.gnt',\n",
       " '666-f.gnt',\n",
       " '057-f.gnt',\n",
       " '526-f.gnt',\n",
       " '674-f.gnt',\n",
       " '1143-f.gnt',\n",
       " '560-f.gnt',\n",
       " '055-f.gnt',\n",
       " '277-f.gnt',\n",
       " '525-f.gnt',\n",
       " '328-f.gnt',\n",
       " '209-f.gnt',\n",
       " '319-f.gnt',\n",
       " '031-f.gnt',\n",
       " '054-f.gnt',\n",
       " '1118-f.gnt',\n",
       " '220-f.gnt',\n",
       " '538-f.gnt',\n",
       " '520-f.gnt',\n",
       " '1080-f.gnt',\n",
       " '1197-f.gnt',\n",
       " '572-f.gnt',\n",
       " '1015-f.gnt',\n",
       " '077-f.gnt',\n",
       " '135-f.gnt',\n",
       " '1056-f.gnt',\n",
       " '401-f.gnt',\n",
       " '085-f.gnt',\n",
       " '344-f.gnt',\n",
       " '047-f.gnt',\n",
       " '589-f.gnt',\n",
       " '201-f.gnt',\n",
       " '564-f.gnt',\n",
       " '684-f.gnt',\n",
       " '642-f.gnt',\n",
       " '1067-f.gnt',\n",
       " '399-f.gnt',\n",
       " '1154-f.gnt',\n",
       " '420-f.gnt',\n",
       " '028-f.gnt',\n",
       " '1106-f.gnt',\n",
       " '534-f.gnt',\n",
       " '1155-f.gnt',\n",
       " '049-f.gnt',\n",
       " '613-f.gnt',\n",
       " '716-f.gnt',\n",
       " '1170-f.gnt',\n",
       " '416-f.gnt',\n",
       " '626-f.gnt',\n",
       " '1160-f.gnt',\n",
       " '263-f.gnt',\n",
       " '1174-f.gnt',\n",
       " '1024-f.gnt',\n",
       " '309-f.gnt',\n",
       " '1237-f.gnt',\n",
       " '592-f.gnt',\n",
       " '502-f.gnt',\n",
       " '1077-f.gnt',\n",
       " '003-f.gnt',\n",
       " '669-f.gnt',\n",
       " '298-f.gnt',\n",
       " '087-f.gnt',\n",
       " '710-f.gnt',\n",
       " '110-f.gnt',\n",
       " '522-f.gnt',\n",
       " '134-f.gnt',\n",
       " '1048-f.gnt',\n",
       " '368-f.gnt',\n",
       " '1050-f.gnt',\n",
       " '1070-f.gnt',\n",
       " '1125-f.gnt',\n",
       " '200-f.gnt',\n",
       " '036-f.gnt',\n",
       " '1058-f.gnt',\n",
       " '618-f.gnt',\n",
       " '605-f.gnt',\n",
       " '215-f.gnt',\n",
       " '065-f.gnt',\n",
       " '720-f.gnt',\n",
       " '404-f.gnt',\n",
       " '289-f.gnt',\n",
       " '1168-f.gnt',\n",
       " '184-f.gnt',\n",
       " '1205-f.gnt',\n",
       " '012-f.gnt',\n",
       " '705-f.gnt',\n",
       " '330-f.gnt',\n",
       " '531-f.gnt',\n",
       " '261-f.gnt',\n",
       " '1128-f.gnt',\n",
       " '089-f.gnt',\n",
       " '1029-f.gnt',\n",
       " '109-f.gnt',\n",
       " '1041-f.gnt',\n",
       " '1020-f.gnt',\n",
       " '1207-f.gnt',\n",
       " '353-f.gnt',\n",
       " '740-f.gnt',\n",
       " '601-f.gnt',\n",
       " '414-f.gnt',\n",
       " '397-f.gnt',\n",
       " '199-f.gnt',\n",
       " '1234-f.gnt',\n",
       " '1138-f.gnt',\n",
       " '034-f.gnt',\n",
       " '214-f.gnt',\n",
       " '1033-f.gnt',\n",
       " '1229-f.gnt',\n",
       " '139-f.gnt',\n",
       " '354-f.gnt',\n",
       " '385-f.gnt',\n",
       " '1181-f.gnt',\n",
       " '329-f.gnt',\n",
       " '655-f.gnt',\n",
       " '259-f.gnt',\n",
       " '1232-f.gnt',\n",
       " '1195-f.gnt',\n",
       " '734-f.gnt',\n",
       " '066-f.gnt',\n",
       " '1153-f.gnt',\n",
       " '171-f.gnt',\n",
       " '722-f.gnt',\n",
       " '631-f.gnt',\n",
       " '704-f.gnt',\n",
       " '1084-f.gnt',\n",
       " '151-f.gnt',\n",
       " '1216-f.gnt',\n",
       " '736-f.gnt',\n",
       " '677-f.gnt',\n",
       " '625-f.gnt',\n",
       " '115-f.gnt',\n",
       " '380-f.gnt',\n",
       " '643-f.gnt',\n",
       " '1064-f.gnt',\n",
       " '1079-f.gnt',\n",
       " '521-f.gnt',\n",
       " '724-f.gnt',\n",
       " '1203-f.gnt',\n",
       " '196-f.gnt',\n",
       " '245-f.gnt',\n",
       " '1202-f.gnt',\n",
       " '088-f.gnt',\n",
       " '537-f.gnt',\n",
       " '023-f.gnt',\n",
       " '079-f.gnt',\n",
       " '1165-f.gnt',\n",
       " '227-f.gnt',\n",
       " '035-f.gnt',\n",
       " '622-f.gnt',\n",
       " '1099-f.gnt',\n",
       " '1066-f.gnt',\n",
       " '233-f.gnt',\n",
       " '523-f.gnt',\n",
       " '1191-f.gnt',\n",
       " '075-f.gnt',\n",
       " '663-f.gnt',\n",
       " '1049-f.gnt',\n",
       " '225-f.gnt',\n",
       " '702-f.gnt',\n",
       " '554-f.gnt',\n",
       " '624-f.gnt',\n",
       " '188-f.gnt',\n",
       " '178-f.gnt',\n",
       " '676-f.gnt',\n",
       " '365-f.gnt',\n",
       " '409-f.gnt',\n",
       " '1231-f.gnt',\n",
       " '239-f.gnt',\n",
       " '1220-f.gnt',\n",
       " '1192-f.gnt',\n",
       " '1127-f.gnt',\n",
       " '518-f.gnt',\n",
       " '405-f.gnt',\n",
       " '683-f.gnt',\n",
       " '362-f.gnt',\n",
       " '1063-f.gnt',\n",
       " '103-f.gnt',\n",
       " '1065-f.gnt',\n",
       " '639-f.gnt',\n",
       " '1095-f.gnt',\n",
       " '1025-f.gnt',\n",
       " '507-f.gnt',\n",
       " '1076-f.gnt',\n",
       " '1142-f.gnt',\n",
       " '1083-f.gnt',\n",
       " '322-f.gnt',\n",
       " '588-f.gnt',\n",
       " '1201-f.gnt',\n",
       " '1071-f.gnt',\n",
       " '370-f.gnt',\n",
       " '349-f.gnt',\n",
       " '120-f.gnt',\n",
       " '223-f.gnt',\n",
       " '519-f.gnt',\n",
       " '546-f.gnt',\n",
       " '1104-f.gnt',\n",
       " '609-f.gnt',\n",
       " '535-f.gnt',\n",
       " '056-f.gnt',\n",
       " '1008-f.gnt',\n",
       " '548-f.gnt',\n",
       " '1132-f.gnt',\n",
       " '591-f.gnt',\n",
       " '133-f.gnt',\n",
       " '001-f.gnt',\n",
       " '623-f.gnt',\n",
       " '1014-f.gnt',\n",
       " '542-f.gnt',\n",
       " '320-f.gnt',\n",
       " '206-f.gnt',\n",
       " '114-f.gnt',\n",
       " '221-f.gnt',\n",
       " '234-f.gnt',\n",
       " '272-f.gnt',\n",
       " '533-f.gnt',\n",
       " '670-f.gnt',\n",
       " '586-f.gnt',\n",
       " '651-f.gnt',\n",
       " '1171-f.gnt',\n",
       " '1006-f.gnt',\n",
       " '657-f.gnt',\n",
       " '699-f.gnt',\n",
       " '1108-f.gnt',\n",
       " '402-f.gnt',\n",
       " '1097-f.gnt',\n",
       " '342-f.gnt',\n",
       " '713-f.gnt',\n",
       " '1144-f.gnt',\n",
       " '600-f.gnt',\n",
       " '374-f.gnt',\n",
       " '418-f.gnt',\n",
       " '636-f.gnt',\n",
       " '1239-f.gnt',\n",
       " '540-f.gnt',\n",
       " '146-f.gnt',\n",
       " '316-f.gnt',\n",
       " '1157-f.gnt',\n",
       " '415-f.gnt',\n",
       " '513-f.gnt',\n",
       " '1037-f.gnt',\n",
       " '686-f.gnt',\n",
       " '117-f.gnt',\n",
       " '074-f.gnt',\n",
       " '413-f.gnt',\n",
       " '1110-f.gnt',\n",
       " '1101-f.gnt',\n",
       " '1178-f.gnt',\n",
       " '164-f.gnt',\n",
       " '379-f.gnt',\n",
       " '1059-f.gnt',\n",
       " '1072-f.gnt',\n",
       " '735-f.gnt',\n",
       " '059-f.gnt',\n",
       " '697-f.gnt',\n",
       " '1087-f.gnt',\n",
       " '1074-f.gnt',\n",
       " '318-f.gnt',\n",
       " '1183-f.gnt',\n",
       " '343-f.gnt',\n",
       " '558-f.gnt',\n",
       " '037-f.gnt',\n",
       " '251-f.gnt',\n",
       " '052-f.gnt',\n",
       " '648-f.gnt',\n",
       " '574-f.gnt',\n",
       " '1211-f.gnt',\n",
       " '632-f.gnt',\n",
       " '638-f.gnt',\n",
       " '1238-f.gnt',\n",
       " '325-f.gnt',\n",
       " '314-f.gnt',\n",
       " '281-f.gnt',\n",
       " '412-f.gnt',\n",
       " '152-f.gnt',\n",
       " '1096-f.gnt',\n",
       " '1133-f.gnt',\n",
       " '043-f.gnt',\n",
       " '707-f.gnt',\n",
       " '527-f.gnt',\n",
       " '267-f.gnt',\n",
       " '703-f.gnt',\n",
       " '222-f.gnt',\n",
       " '1109-f.gnt',\n",
       " '501-f.gnt',\n",
       " '417-f.gnt',\n",
       " '1204-f.gnt',\n",
       " '1032-f.gnt',\n",
       " '131-f.gnt',\n",
       " '099-f.gnt',\n",
       " '575-f.gnt',\n",
       " '672-f.gnt',\n",
       " '258-f.gnt',\n",
       " '356-f.gnt',\n",
       " '177-f.gnt',\n",
       " '604-f.gnt',\n",
       " '687-f.gnt',\n",
       " '1021-f.gnt',\n",
       " '1060-f.gnt',\n",
       " '1126-f.gnt',\n",
       " '1116-f.gnt',\n",
       " '378-f.gnt',\n",
       " '269-f.gnt',\n",
       " '366-f.gnt',\n",
       " '634-f.gnt',\n",
       " '1120-f.gnt',\n",
       " '717-f.gnt',\n",
       " '039-f.gnt',\n",
       " '1069-f.gnt',\n",
       " '236-f.gnt',\n",
       " '579-f.gnt',\n",
       " '1011-f.gnt',\n",
       " '633-f.gnt',\n",
       " '165-f.gnt',\n",
       " '1219-f.gnt',\n",
       " '305-f.gnt',\n",
       " '1103-f.gnt',\n",
       " '1189-f.gnt',\n",
       " '279-f.gnt',\n",
       " '387-f.gnt',\n",
       " '407-f.gnt',\n",
       " '561-f.gnt',\n",
       " '1210-f.gnt',\n",
       " '555-f.gnt',\n",
       " '1111-f.gnt',\n",
       " '506-f.gnt',\n",
       " '207-f.gnt',\n",
       " '1027-f.gnt',\n",
       " '1030-f.gnt',\n",
       " '208-f.gnt',\n",
       " '608-f.gnt',\n",
       " '514-f.gnt',\n",
       " '550-f.gnt',\n",
       " '048-f.gnt',\n",
       " '1150-f.gnt',\n",
       " '027-f.gnt',\n",
       " '016-f.gnt',\n",
       " '101-f.gnt',\n",
       " '549-f.gnt',\n",
       " '091-f.gnt',\n",
       " '1163-f.gnt',\n",
       " '159-f.gnt',\n",
       " '566-f.gnt',\n",
       " '044-f.gnt',\n",
       " '291-f.gnt',\n",
       " '1090-f.gnt',\n",
       " '649-f.gnt',\n",
       " '313-f.gnt',\n",
       " '616-f.gnt',\n",
       " '1226-f.gnt',\n",
       " '678-f.gnt',\n",
       " '612-f.gnt',\n",
       " '384-f.gnt',\n",
       " '180-f.gnt',\n",
       " '360-f.gnt',\n",
       " '1086-f.gnt',\n",
       " '577-f.gnt',\n",
       " '081-f.gnt',\n",
       " '292-f.gnt',\n",
       " '578-f.gnt',\n",
       " '640-f.gnt',\n",
       " '645-f.gnt',\n",
       " '1081-f.gnt',\n",
       " '647-f.gnt',\n",
       " '067-f.gnt',\n",
       " '090-f.gnt',\n",
       " '671-f.gnt',\n",
       " '1004-f.gnt',\n",
       " '681-f.gnt',\n",
       " '1005-f.gnt',\n",
       " '051-f.gnt',\n",
       " '630-f.gnt',\n",
       " '078-f.gnt',\n",
       " '315-f.gnt',\n",
       " '1117-f.gnt',\n",
       " '364-f.gnt',\n",
       " '1026-f.gnt',\n",
       " '1193-f.gnt',\n",
       " '652-f.gnt',\n",
       " '243-f.gnt',\n",
       " '1139-f.gnt',\n",
       " '167-f.gnt',\n",
       " '503-f.gnt',\n",
       " '007-f.gnt',\n",
       " '1176-f.gnt',\n",
       " '018-f.gnt',\n",
       " '372-f.gnt',\n",
       " '706-f.gnt',\n",
       " '288-f.gnt',\n",
       " '1113-f.gnt',\n",
       " '170-f.gnt',\n",
       " '123-f.gnt',\n",
       " '1164-f.gnt',\n",
       " '286-f.gnt',\n",
       " '355-f.gnt',\n",
       " '026-f.gnt',\n",
       " '310-f.gnt',\n",
       " '010-f.gnt',\n",
       " '041-f.gnt',\n",
       " '192-f.gnt',\n",
       " '565-f.gnt',\n",
       " '620-f.gnt',\n",
       " '1031-f.gnt',\n",
       " '602-f.gnt',\n",
       " '505-f.gnt',\n",
       " '1073-f.gnt',\n",
       " '1175-f.gnt',\n",
       " '072-f.gnt',\n",
       " '1173-f.gnt',\n",
       " '1208-f.gnt',\n",
       " '203-f.gnt',\n",
       " '1035-f.gnt',\n",
       " '1148-f.gnt',\n",
       " '280-f.gnt',\n",
       " '1223-f.gnt',\n",
       " '659-f.gnt',\n",
       " '689-f.gnt',\n",
       " '730-f.gnt',\n",
       " '338-f.gnt',\n",
       " '287-f.gnt',\n",
       " '1130-f.gnt',\n",
       " '266-f.gnt',\n",
       " '1158-f.gnt',\n",
       " '155-f.gnt',\n",
       " '274-f.gnt',\n",
       " '1227-f.gnt',\n",
       " '1036-f.gnt',\n",
       " '004-f.gnt',\n",
       " '185-f.gnt',\n",
       " '296-f.gnt',\n",
       " '1122-f.gnt',\n",
       " '183-f.gnt',\n",
       " '637-f.gnt',\n",
       " '556-f.gnt',\n",
       " '690-f.gnt',\n",
       " '679-f.gnt',\n",
       " '712-f.gnt',\n",
       " '339-f.gnt',\n",
       " '190-f.gnt',\n",
       " '191-f.gnt',\n",
       " '327-f.gnt',\n",
       " '1039-f.gnt',\n",
       " '1016-f.gnt',\n",
       " '321-f.gnt',\n",
       " '1100-f.gnt',\n",
       " '1147-f.gnt',\n",
       " '619-f.gnt',\n",
       " '590-f.gnt',\n",
       " '610-f.gnt',\n",
       " '1134-f.gnt',\n",
       " '002-f.gnt',\n",
       " '530-f.gnt',\n",
       " '273-f.gnt',\n",
       " '403-f.gnt',\n",
       " '739-f.gnt',\n",
       " '723-f.gnt',\n",
       " '627-f.gnt',\n",
       " '1177-f.gnt',\n",
       " '198-f.gnt',\n",
       " '179-f.gnt',\n",
       " '656-f.gnt',\n",
       " '1186-f.gnt',\n",
       " '580-f.gnt',\n",
       " '1182-f.gnt',\n",
       " '628-f.gnt',\n",
       " '094-f.gnt',\n",
       " '1091-f.gnt',\n",
       " '1092-f.gnt',\n",
       " '275-f.gnt',\n",
       " '393-f.gnt',\n",
       " '083-f.gnt',\n",
       " '1001-f.gnt',\n",
       " '545-f.gnt',\n",
       " '1213-f.gnt',\n",
       " '559-f.gnt',\n",
       " '182-f.gnt',\n",
       " '576-f.gnt',\n",
       " '217-f.gnt',\n",
       " '1166-f.gnt',\n",
       " '357-f.gnt',\n",
       " '1156-f.gnt',\n",
       " '696-f.gnt',\n",
       " '504-f.gnt',\n",
       " '307-f.gnt',\n",
       " '271-f.gnt',\n",
       " '107-f.gnt',\n",
       " '1196-f.gnt',\n",
       " '014-f.gnt',\n",
       " '532-f.gnt',\n",
       " '098-f.gnt',\n",
       " '064-f.gnt',\n",
       " '1180-f.gnt',\n",
       " '661-f.gnt',\n",
       " '729-f.gnt',\n",
       " '1055-f.gnt',\n",
       " '528-f.gnt',\n",
       " '721-f.gnt',\n",
       " '383-f.gnt',\n",
       " '162-f.gnt',\n",
       " '1224-f.gnt',\n",
       " '541-f.gnt',\n",
       " '140-f.gnt',\n",
       " '662-f.gnt',\n",
       " '1093-f.gnt',\n",
       " '238-f.gnt',\n",
       " '646-f.gnt',\n",
       " '228-f.gnt',\n",
       " '331-f.gnt',\n",
       " '144-f.gnt',\n",
       " '138-f.gnt',\n",
       " '1169-f.gnt',\n",
       " '025-f.gnt',\n",
       " '653-f.gnt',\n",
       " '1159-f.gnt',\n",
       " '719-f.gnt',\n",
       " '1136-f.gnt',\n",
       " '1013-f.gnt',\n",
       " '1215-f.gnt',\n",
       " '015-f.gnt',\n",
       " '694-f.gnt',\n",
       " '570-f.gnt',\n",
       " '033-f.gnt',\n",
       " '019-f.gnt',\n",
       " '711-f.gnt',\n",
       " '738-f.gnt',\n",
       " '1075-f.gnt',\n",
       " '1152-f.gnt',\n",
       " '145-f.gnt',\n",
       " '1082-f.gnt',\n",
       " '512-f.gnt',\n",
       " '1179-f.gnt',\n",
       " '581-f.gnt',\n",
       " '1235-f.gnt',\n",
       " '1149-f.gnt',\n",
       " '728-f.gnt',\n",
       " '1046-f.gnt',\n",
       " '070-f.gnt',\n",
       " '562-f.gnt',\n",
       " '128-f.gnt',\n",
       " '1184-f.gnt',\n",
       " '303-f.gnt',\n",
       " '1119-f.gnt',\n",
       " '1185-f.gnt',\n",
       " '584-f.gnt',\n",
       " '358-f.gnt',\n",
       " '119-f.gnt',\n",
       " '264-f.gnt',\n",
       " '1221-f.gnt',\n",
       " '294-f.gnt',\n",
       " '517-f.gnt',\n",
       " '017-f.gnt',\n",
       " '284-f.gnt',\n",
       " '324-f.gnt',\n",
       " '009-f.gnt',\n",
       " '166-f.gnt',\n",
       " '332-f.gnt',\n",
       " '375-f.gnt',\n",
       " '149-f.gnt',\n",
       " '737-f.gnt',\n",
       " '1218-f.gnt',\n",
       " '596-f.gnt',\n",
       " '197-f.gnt',\n",
       " '1146-f.gnt',\n",
       " '1209-f.gnt',\n",
       " '621-f.gnt',\n",
       " '1151-f.gnt',\n",
       " '299-f.gnt',\n",
       " '557-f.gnt',\n",
       " '255-f.gnt',\n",
       " '603-f.gnt',\n",
       " '726-f.gnt',\n",
       " '100-f.gnt',\n",
       " '212-f.gnt',\n",
       " '260-f.gnt',\n",
       " '515-f.gnt',\n",
       " '390-f.gnt',\n",
       " '1088-f.gnt',\n",
       " '395-f.gnt',\n",
       " '573-f.gnt',\n",
       " '1225-f.gnt',\n",
       " '186-f.gnt',\n",
       " '237-f.gnt',\n",
       " '1054-f.gnt',\n",
       " '1003-f.gnt',\n",
       " '1240-f.gnt',\n",
       " '658-f.gnt',\n",
       " '1028-f.gnt',\n",
       " '606-f.gnt',\n",
       " '685-f.gnt',\n",
       " '367-f.gnt',\n",
       " '216-f.gnt',\n",
       " '1115-f.gnt',\n",
       " '1045-f.gnt',\n",
       " '587-f.gnt',\n",
       " '571-f.gnt',\n",
       " '334-f.gnt',\n",
       " '691-f.gnt',\n",
       " '389-f.gnt',\n",
       " '1206-f.gnt',\n",
       " '1023-f.gnt',\n",
       " '1038-f.gnt',\n",
       " '544-f.gnt',\n",
       " '211-f.gnt',\n",
       " '692-f.gnt',\n",
       " '290-f.gnt',\n",
       " '650-f.gnt',\n",
       " '306-f.gnt',\n",
       " '175-f.gnt',\n",
       " '529-f.gnt',\n",
       " '508-f.gnt',\n",
       " '709-f.gnt',\n",
       " '021-f.gnt',\n",
       " '143-f.gnt',\n",
       " '543-f.gnt',\n",
       " '340-f.gnt',\n",
       " '270-f.gnt',\n",
       " '080-f.gnt',\n",
       " '359-f.gnt',\n",
       " '1062-f.gnt',\n",
       " '1112-f.gnt',\n",
       " '727-f.gnt',\n",
       " '247-f.gnt',\n",
       " '106-f.gnt',\n",
       " '1141-f.gnt',\n",
       " '369-f.gnt',\n",
       " '122-f.gnt',\n",
       " '218-f.gnt',\n",
       " '1052-f.gnt',\n",
       " '1044-f.gnt',\n",
       " '127-f.gnt',\n",
       " '163-f.gnt',\n",
       " '1018-f.gnt',\n",
       " '204-f.gnt',\n",
       " '392-f.gnt',\n",
       " '061-f.gnt',\n",
       " '311-f.gnt',\n",
       " '350-f.gnt',\n",
       " '536-f.gnt',\n",
       " '141-f.gnt',\n",
       " '092-f.gnt',\n",
       " '552-f.gnt',\n",
       " '731-f.gnt',\n",
       " '569-f.gnt',\n",
       " '249-f.gnt',\n",
       " '598-f.gnt',\n",
       " '040-f.gnt',\n",
       " '701-f.gnt',\n",
       " '1145-f.gnt',\n",
       " '1057-f.gnt',\n",
       " '230-f.gnt',\n",
       " '1121-f.gnt',\n",
       " '045-f.gnt',\n",
       " '644-f.gnt',\n",
       " '1089-f.gnt',\n",
       " '1228-f.gnt',\n",
       " '347-f.gnt',\n",
       " '210-f.gnt',\n",
       " '693-f.gnt',\n",
       " '718-f.gnt',\n",
       " '660-f.gnt',\n",
       " '664-f.gnt',\n",
       " '148-f.gnt',\n",
       " '348-f.gnt',\n",
       " '1123-f.gnt',\n",
       " '096-f.gnt',\n",
       " '1198-f.gnt',\n",
       " '063-f.gnt',\n",
       " '011-f.gnt',\n",
       " '1105-f.gnt',\n",
       " '231-f.gnt',\n",
       " '154-f.gnt',\n",
       " '1194-f.gnt',\n",
       " '068-f.gnt',\n",
       " '1162-f.gnt',\n",
       " '093-f.gnt',\n",
       " '665-f.gnt',\n",
       " '105-f.gnt',\n",
       " '176-f.gnt',\n",
       " '1009-f.gnt',\n",
       " '333-f.gnt',\n",
       " '1190-f.gnt']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_of_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "charset_without_accent = [\n",
    "    \"a\",\n",
    "    \"b\",\n",
    "    \"c\",\n",
    "    \"d\",\n",
    "    \"e\",\n",
    "    \"f\",\n",
    "    \"g\",\n",
    "    \"h\",\n",
    "    \"i\",\n",
    "    \"j\",\n",
    "    \"k\",\n",
    "    \"l\",\n",
    "    \"m\",\n",
    "    \"n\",\n",
    "    \"o\",\n",
    "    \"p\",\n",
    "    \"q\",\n",
    "    \"r\",\n",
    "    \"s\",\n",
    "    \"t\",\n",
    "    \"u\",\n",
    "    \"v\",\n",
    "    \"w\",\n",
    "    \"x\",\n",
    "    \"y\",\n",
    "    \"z\",\n",
    "    \"A\",\n",
    "    \"B\",\n",
    "    \"C\",\n",
    "    \"D\",\n",
    "    \"E\",\n",
    "    \"F\",\n",
    "    \"G\",\n",
    "    \"H\",\n",
    "    \"I\",\n",
    "    \"J\",\n",
    "    \"K\",\n",
    "    \"L\",\n",
    "    \"M\",\n",
    "    \"N\",\n",
    "    \"O\",\n",
    "    \"P\",\n",
    "    \"Q\",\n",
    "    \"R\",\n",
    "    \"S\",\n",
    "    \"T\",\n",
    "    \"U\",\n",
    "    \"V\",\n",
    "    \"W\",\n",
    "    \"X\",\n",
    "    \"Y\",\n",
    "    \"Z\",\n",
    "    \"0\",\n",
    "    \"1\",\n",
    "    \"2\",\n",
    "    \"3\",\n",
    "    \"4\",\n",
    "    \"5\",\n",
    "    \"6\",\n",
    "    \"7\",\n",
    "    \"8\",\n",
    "    \"9\",\n",
    "    \"!\",\n",
    "    \"?\",\n",
    "]\n",
    "symbols = [\n",
    "    '\"',\n",
    "    \"#\",\n",
    "    \"$\",\n",
    "    \"%\",\n",
    "    \"&\",\n",
    "    \"'\",\n",
    "    \"(\",\n",
    "    \")\",\n",
    "    \"*\",\n",
    "    \"+\",\n",
    "    \",\",\n",
    "    \"-\",\n",
    "    \".\",\n",
    "    \"/\",\n",
    "    \":\",\n",
    "    \";\",\n",
    "    \"<\",\n",
    "    \"=\",\n",
    "    \">\",\n",
    "    \"@\",\n",
    "    \"[\",\n",
    "    \"\\\\\",\n",
    "    \"]\",\n",
    "    \"^\",\n",
    "    \"_\",\n",
    "    \"`\",\n",
    "    \"{\",\n",
    "    \"|\",\n",
    "    \"}\",\n",
    "    \"~\",\n",
    "    \" \",\n",
    "]\n",
    "accent_charset = [\n",
    "    \"à\",\n",
    "    \"á\",\n",
    "    \"â\",\n",
    "    \"ã\",\n",
    "    \"ä\",\n",
    "    \"å\",\n",
    "    \"ā\",\n",
    "    \"æ\",\n",
    "    \"ç\",\n",
    "    \"è\",\n",
    "    \"é\",\n",
    "    \"ê\",\n",
    "    \"ë\",\n",
    "    \"ì\",\n",
    "    \"í\",\n",
    "    \"î\",\n",
    "    \"ï\",\n",
    "    \"ð\",\n",
    "    \"ñ\",\n",
    "    \"ò\",\n",
    "    \"ó\",\n",
    "    \"ô\",\n",
    "    \"õ\",\n",
    "    \"ö\",\n",
    "    \"ō\",\n",
    "    \"ø\",\n",
    "    \"ù\",\n",
    "    \"ú\",\n",
    "    \"û\",\n",
    "    \"ü\",\n",
    "    \"ý\",\n",
    "    \"þ\",\n",
    "    \"ÿ\",\n",
    "    \"À\",\n",
    "    \"Á\",\n",
    "    \"Â\",\n",
    "    \"Ã\",\n",
    "    \"Ä\",\n",
    "    \"Å\",\n",
    "    \"Æ\",\n",
    "    \"Ç\",\n",
    "    \"È\",\n",
    "    \"É\",\n",
    "    \"Ê\",\n",
    "    \"Ë\",\n",
    "    \"Ì\",\n",
    "    \"Í\",\n",
    "    \"Î\",\n",
    "    \"Ï\",\n",
    "    \"Ð\",\n",
    "    \"Ñ\",\n",
    "    \"Ò\",\n",
    "    \"Ó\",\n",
    "    \"Ô\",\n",
    "    \"Õ\",\n",
    "    \"Ö\",\n",
    "    \"Ø\",\n",
    "    \"Ù\",\n",
    "    \"Ú\",\n",
    "    \"Û\",\n",
    "    \"Ü\",\n",
    "    \"Ý\",\n",
    "    \"Þ\",\n",
    "    \"Ÿ\",\n",
    "]\n",
    "weird_charset = [\"«\", \"»\", \"—\", \"’\", \"°\", \"–\", \"œ\"]\n",
    "charset = charset_without_accent + accent_charset + weird_charset + symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "with open(os.path.join(\"default_charset.json\"), \"w\") as f:\n",
    "    json.dump(charset, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "with open(os.path.join(\"default_charset_without_accent.json\"), \"w\") as f:\n",
    "    json.dump(charset_without_accent, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "charset_chinse = pickle.load(open(\"/home/rbaena/projects/OCR/DINO/charset_HWDB.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "charset_chinse == pickle.load(open(\"/home/rbaena/projects/OCR/DINO/charset_HWDB.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
